<?
/*
 *  Para instalar PHP en linux, desde la consola:
 * 	$ aptitude install php5-cli
 * 
 *  Para instalar libreria cURL de PHP que usa este script:
 * 	$ aptitude install php5-curl
 * 
 *  Para correr este script:
 * 	$ php crawl.gutenberg.php
 * 
 *  A medida que va obteniendo nuevos textos, los va a ir agregando en 
 *  el archivo 'finalText.txt' que se creará en el mismo directorio.
 * 
 *	Ahora mismo esta configurado para obtener todos los textos EN INGLES
 *	en los listados de titulos A, B y C. Si se desea, se pueden agregar 
 * 	más.
 */
 
if( !function_exists('curl_init') ) die('cURL no esta instalado!'.PHP_EOL);

$filename_text = "./finalText.txt";
$output_file = fopen($filename_text , 'a');

$seeds = Array();
$seeds[] = "http://www.gutenberg.org/browse/titles/a";
$seeds[] = "http://www.gutenberg.org/browse/titles/b";
$seeds[] = "http://www.gutenberg.org/browse/titles/c";

// Matchea solo para textos en ingles 
$preg_item = '#href="/ebooks/(?P<BookID>\d+)"[^>]*>.+?</a> *\(English#si';

foreach( $seeds as $seed ){
	echo PHP_EOL . "Crawling seed: '$seed'..." . PHP_EOL . PHP_EOL;
	
	echo "Obteniendo HTML..." . PHP_EOL . PHP_EOL;
	$seed_html_code = downloadSourceCode($seed);
	
	if( strlen($seed_html_code) == 0  ){
		echo "Error: No se obtuvo HTML de seed '$seed'" . PHP_EOL;
		continue;
	}
	
	if( preg_match_all( $preg_item , $seed_html_code , $matches ) ){
		
		for( $i=0 ; $i<count($matches[0]) ; $i++ ){
		
			$BookID = $matches['BookID'][$i];
			$BookUrl = get_book_url($BookID);
			
			
			echo "Chequeando si url '$BookUrl' esta online..." . PHP_EOL; flush();
			if( !check_if_url_is_online($BookUrl) ){
				echo "No se pudo obtener texto de '$BookUrl'" . PHP_EOL . PHP_EOL; flush(); 
				continue;
			}
			
			echo "Bajando " . $BookUrl.PHP_EOL.PHP_EOL; flush();
			
			downloadText($BookUrl,$output_file);
		}
	}
}

fclose($output_file);

echo PHP_EOL . PHP_EOL . "Proceso finalizado!" . PHP_EOL;






function get_book_url($BookID){
	// Aca hay un listado de mirrors que se pueden utilizar:
	// http://www.gutenberg.org/MIRRORS.ALL
	
	assert( is_string($BookID) );
	$final = "http://mirrors.xmission.com/gutenberg/";
	for($i=0 ; $i<strlen($BookID)-1 ; $i++){
		$final .= $BookID[$i] . '/';
	}
	$final .= "$BookID/$BookID.txt";
	
	return $final;
}

function check_if_url_is_online($url){
	$url = @parse_url($url);
	if (!$url) return false;

	$url = array_map('trim', $url);
	$url['port'] = (!isset($url['port'])) ? 80 : (int)$url['port'];

	$path = (isset($url['path'])) ? $url['path'] : '/';
	$path .= (isset($url['query'])) ? "?$url[query]" : '';

	if (isset($url['host']) && $url['host'] != gethostbyname($url['host'])) {

		$fp = fsockopen($url['host'], $url['port'], $errno, $errstr, 30);

		if (!$fp) return false; //socket not opened

		fputs($fp, "HEAD $path HTTP/1.1\r\nHost: $url[host]\r\n\r\n"); //socket opened
		$headers = fread($fp, 4096);
		fclose($fp);

		if(preg_match('#^HTTP/.*\s+[(200|301|302)]+\s#i', $headers)){//matching header
			return true;
		}
		else return false;

	} // if parse url
	else return false;
}

function downloadSourceCode($url){
	$ch = curl_init();
	curl_setopt($ch, CURLOPT_URL, $url );
	curl_setopt($ch, CURLOPT_USERAGENT, "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.0.1) Gecko/2008070208 Firefox/3.0.1");
	curl_setopt($ch, CURLOPT_HTTPHEADER, array("Accept-Language: es-es,en"));  
	curl_setopt($ch, CURLOPT_POST,false);  
	curl_setopt($ch, CURLOPT_TIMEOUT, 40);
	curl_setopt($ch, CURLOPT_REFERER, 'http://www.google.com');
	curl_setopt($ch, CURLOPT_FOLLOWLOCATION, true);
	curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);
	
	$result = curl_exec ($ch);
	curl_close($ch);
	
	return $result;
}

function downloadText($url,$file_handle){
	$ch = curl_init();
	curl_setopt($ch, CURLOPT_URL, $url );
	curl_setopt($ch, CURLOPT_TIMEOUT, 40);
	curl_setopt($ch, CURLOPT_FOLLOWLOCATION, true);
	curl_setopt($ch, CURLOPT_FILE, $file_handle);
	
	curl_exec ($ch);
	curl_close($ch);
}
?>
